package edu.isi.pedwork.topic;

import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.io.*;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.w3c.dom.Document;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

// author KJ

@SuppressWarnings("unchecked")
public class LabelModel {

	static String CLEANINGLIST[] = { "[0-9]+", "[0-9]\\.", "\\*", "\\$", "\\,",
			"\\.", "\\'s", "\"", "\\'", "\\]", "\\{", "\\}", "/quote",
			"\\[ /quote", "//", "\\[ b", "\\[", "\\(", "\\)", "--", ">", "=",
			"yes", "Yes", "Hi", "!", "u", "hi", "you", "ok", "thx", "ya",
			"yes", "Thanks", "http", "\\~", "~", "bye", "hello", "help",
			"need", "\\+" };

	static String semester[] = { "20073_1", "20073_2", "20073_3", "20073_4",
			"20093_1", "20093_2", "20093_3", "20093_4" };

	public static void main(String args[]) throws Exception {

		String semester[] = { "20061", "20062", "20063", "20071", "20073",
				"20081", "20082", "20083", "20091", "20092", "20093" };
		String workingdir = "D:\\Documents\\USC\\ISI\\topic\\labelledlda\\KJ's code\\Archive\\";

		Hashtable specificTermList = new Hashtable();
		Hashtable LabelBagOfWords = getAllLabels(workingdir);
		write_label_model(LabelBagOfWords, workingdir + "label.txt");
	}

	public static String join(Collection s, String delimiter) {
		StringBuffer buffer = new StringBuffer();
		Iterator iter = s.iterator();
		if (iter.hasNext()) {
			buffer.append(iter.next());
			while (iter.hasNext()) {
				buffer.append(delimiter);
				buffer.append(iter.next());
			}
		}
		return buffer.toString();
	}

	public static void write_label_model(
			Hashtable<String, Hashtable<String, Integer>> labelBagWords,
			String path) throws IOException {
		FileWriter writer = new FileWriter(path);
		Enumeration<String> itr = labelBagWords.keys();
		for (; itr.hasMoreElements();) {
			String label = itr.nextElement();
			Hashtable<String, Integer> set = labelBagWords.get(label);
			String words = join((Collection) set.keySet(), ",");
			writer.write(label + "\t" + words + "\n");
		}
	}

	public static Hashtable getAllLabels(String path) {
		Hashtable LabelBagOfWords = new Hashtable();

		String[] LABELS = { "TS", "LCV", "DIS", "MC", "MS", "RPC", "SC", "AR",
				"WSB", "FSC", "MM", "VM", "MP", "TLB", "PRO", "TEST", "DOC",
				"SUBMIT", "NAC", "SIM" };
		Hashtable ParentLabel = new Hashtable();
		ParentLabel.put("LCV", "TS");
		ParentLabel.put("RPC", "DIS");
		ParentLabel.put("MC", "DIS");
		ParentLabel.put("MS", "DIS");
		ParentLabel.put("AR", "SC");
		ParentLabel.put("WSB", "SC");
		ParentLabel.put("FSC", "SC");
		ParentLabel.put("VM", "MM");
		ParentLabel.put("MP", "MM");
		ParentLabel.put("TLB", "MM");

		for (int i = 0; i < LABELS.length; i++)
			LabelBagOfWords.put(LABELS[i], new Hashtable());
		try {

			for (int j = 0; j < semester.length; j++) {
				Document xmlDoc = createDomParser(path + semester[j] + ".xml");
				for (int i = 0; i < LABELS.length; i++)
					LabelBagOfWords = readXMLData(xmlDoc, LABELS[i],
							LabelBagOfWords);
			}

		} catch (Exception ee) {
			System.out.println(ee);
		}
		try {
			Document xmlDoc = createDomParser(path + "/Discussion.xml");
			for (int i = 0; i < LABELS.length; i++)
				LabelBagOfWords = readXMLData(xmlDoc, LABELS[i],
						LabelBagOfWords);
		} catch (Exception ee) {
			System.out.println(ee);
		}

		Hashtable LabelBOWnew = new Hashtable();
		for (int i = 0; i < LABELS.length; i++) {
			LabelBOWnew.put(LABELS[i], new Hashtable());
		}

		// LabelBagOfWords term clean and n-gram double count
		Iterator iter = LabelBagOfWords.keySet().iterator();
		while (iter.hasNext()) {
			String curLabel = (String) iter.next();
			Hashtable curHT = (Hashtable) LabelBagOfWords.get(curLabel);
			Iterator iter2 = curHT.keySet().iterator();

			Hashtable newHT = (Hashtable) LabelBOWnew.get(curLabel);

			while (iter2.hasNext()) {
				String oriTerm = (String) iter2.next();
				String curTerm = oriTerm;
				Integer count = (Integer) curHT.get(oriTerm);
				String words[] = curTerm.trim().split(" ");
				if (words.length > 1) {
					for (int fromidx = 0; fromidx < words.length; fromidx++) {
						for (int toidx = words.length - 1; toidx >= fromidx; toidx--) {
							String cmbStr = words[fromidx];
							for (int i = fromidx + 1; i <= toidx; i++) {
								cmbStr = cmbStr + " " + words[i];
							}
							newHT = insertTerm(newHT, cmbStr, count);
							if (ParentLabel.containsKey(curLabel)) {
								String parentLabel = (String) ParentLabel
										.get(curLabel);
								Hashtable newParentHT = (Hashtable) LabelBOWnew
										.get(parentLabel);
								LabelBOWnew.put(parentLabel, insertTerm(
										newParentHT, cmbStr, count));
							}
						}
					}
				} else {
					newHT = insertTerm(newHT, curTerm, count);
					if (ParentLabel.containsKey(curLabel)) {
						String parentLabel = (String) ParentLabel.get(curLabel);
						Hashtable newParentHT = (Hashtable) LabelBOWnew
								.get(parentLabel);
						LabelBOWnew.put(parentLabel, insertTerm(newParentHT,
								curTerm, count));
					}
				}
			}
			LabelBOWnew.put(curLabel, newHT);

		}

		return LabelBOWnew;
	}

	public static Hashtable insertTerm(Hashtable termtable, String t, Integer k) {
		if (termtable.containsKey(t)) {
			Integer i = (Integer) termtable.get(t);
			termtable.put(t, (Integer) i + k);
		} else {
			termtable.put(t, (Integer) k);
		}
		return termtable;
	}

	// term will clean
	private static Hashtable readXMLData(Document xmlDoc, String TAG,
			Hashtable LabelTable) {
		try {
			if (!(xmlDoc == null)) {
				NodeList LabelList = xmlDoc.getElementsByTagName(TAG);
				int num = LabelList.getLength();
				Hashtable hs = (Hashtable) LabelTable.get(TAG);
				for (int i = 1; i < num; i++) {
					String labelstr = LabelList.item(i).getTextContent().trim();

					hs = hashtableInsert(hs, labelstr);
				}

				LabelTable.put(TAG, hs);

			}
		} catch (Exception me) {
			System.out.println("exception in readxml : " + me.getCause());
		}
		return LabelTable;
	}

	private static Hashtable hashtableInsert(Hashtable hs, String labelstr) {
		if (hs.containsKey(labelstr)) {
			Integer count = (Integer) hs.get(labelstr);
			hs.put(labelstr, count + 1);
		} else {
			hs.put(labelstr, 1);
		}
		return hs;
	}

	private static Document createDomParser(String fileName) {
		Document xmlDoc = null;

		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
		factory.setIgnoringElementContentWhitespace(true);
		DocumentBuilder builder = null;

		try {
			builder = factory.newDocumentBuilder();
		} catch (ParserConfigurationException e) {
			e.printStackTrace();
			return null;
		}

		try {
			xmlDoc = builder.parse(new File(fileName));

		} catch (SAXException e) {
			e.printStackTrace();
			return null;
		} catch (IOException e) {
			e.printStackTrace();
			return null;
		}
		return xmlDoc;
	}

	public static Hashtable insertTerm(Hashtable termtable, String t) {
		if (termtable.containsKey(t)) {
			Integer i = (Integer) termtable.get(t);
			termtable.put(t, (Integer) i + 1);
		} else {
			termtable.put(t, (Integer) 1);
		}
		return termtable;
	}

	public static void loadSaveFile_LLDAVer(String folder, String outfileName,
			String outputBaseDir, Hashtable semesters, Hashtable label) {

		Hashtable labelHS = new Hashtable();

		Iterator Labeliter = label.keySet().iterator();
		while (Labeliter.hasNext()) {
			String labelstr = (String) Labeliter.next();
			Set bow = (Set) (((Hashtable) label.get(labelstr)).keySet());
			labelHS.put(labelstr, bow);
		}

		try {

			PrintWriter out = new PrintWriter(new FileOutputStream(outfileName));

			LineNumberReader lr = null;
			String curLine = "";

			String semester[] = { "20061", "20062", "20063", "20071", "20073",
					"20081", "20082", "20083", "20091", "20092", "20093" };
			String project[] = { "Project_1_", "Project_2_", "Project_3_",
					"Project_4_" };

			for (Integer i = 0; i < semester.length; i++) {
				for (Integer ii = 0; ii < project.length; ii++) {
					String fileName = folder + project[ii] + "csci402_"
							+ semester[i] + "_feature_clrd.txt";

					lr = new LineNumberReader(new FileReader(fileName));

					while ((curLine = lr.readLine()) != null) {
						String thisTerm = "";
						HashSet thisLabel = new HashSet();
						String words[] = curLine.trim().split(":");
						if (words.length > 1) {
							String terms[] = words[1].trim().split(",");
							for (int j = 0; j < terms.length; j++) {

								// term insert in structure
								String tmp = "";
								StringTokenizer st = new StringTokenizer(
										terms[j]);

								while (st.hasMoreTokens()) {
									if (tmp.equals(""))
										tmp = st.nextToken().toLowerCase();
									else
										tmp = tmp + "_"
												+ st.nextToken().toLowerCase(); // connect
									// one
									// term
									// using
									// under
									// bar
								}
								thisLabel = LabelSetUpdate(thisLabel, labelHS,
										terms[j]);
								// System.out.println(tmp);
								thisTerm = thisTerm + " " + tmp;

							}
						}
						String thisLabelStr = "";
						Iterator iter = thisLabel.iterator();
						while (iter.hasNext()) {
							if (thisLabelStr.equals(""))
								thisLabelStr = (String) iter.next();
							else
								thisLabelStr = thisLabelStr + " "
										+ (String) iter.next();
						}
						// and store in cvs file
						if (!thisTerm.equals(""))
							out.println(semester[i] + "\t" + project[ii] + "\t"
									+ words[0].trim() + "\t" + thisLabelStr
									+ "\t" + thisTerm);
					}
				}
			}
			out.close();
		} catch (Exception e) {
			System.out.println(e);
		}
	}

	public static HashSet LabelSetUpdate(HashSet thisLabel, Hashtable labelHS,
			String term) {
		Iterator iter = labelHS.keySet().iterator();
		while (iter.hasNext()) {
			String curLabel = (String) iter.next();
			if (((Set) labelHS.get(curLabel)).contains(term)) {
				thisLabel.add(curLabel);
			}

		}
		return thisLabel;
	}

	private static int patternmatcher(String PATTERN[], String INPUT) {
		int seq = -1;
		for (int i = 0; i < PATTERN.length; i++) {
			Pattern p = Pattern.compile(PATTERN[i]);
			Matcher m = p.matcher(INPUT);
			if (m.matches()) {
				seq = i;
				return (seq);
			}
		}
		return (seq);

	}
}
